import scrapy
from lxml import etree
import json
from pymongo import MongoClient
# from ..items import TutorialItem
import  random
import hashlib

class ShirenSpider(scrapy.Spider):
    name ="shiren"

    def start_requests(self):
        # scrapy.log.start()  /chaxun/zuozhe/13046.html

        urls = ['http://www.shicimingju.com/chaxun/zuozhe/13046.html',]
        # for i in range(1, 46):
        #     urls.append("http://www.shicimingju.com/chaxun/zuozhe/"+str(i)+".html")
        print(urls)

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):

        pagination = response.css('.pagination a::attr(href)').getall()

        if pagination is not None:
            plen = (len(pagination)) - 1
            newHref = 'http://www.shicimingju.com' + str(pagination[plen])
            print(newHref)
            yield scrapy.Request(url=newHref, callback=self.parse)

        item = response.css('.layui-col-sm7 div')
        for i in item:
            href = i.css('h3 a::attr(href)').getall()
            text = i.css('h3::text').getall()

            if text is not None:
                print(text)
            # title = i.css('div a::text').extract_first()
            if href is not None:
                newHref = 'http://www.shicimingju.com' + str(href)
                yield scrapy.Request(url=newHref, callback=self.getDetail)

    def getDetail(self, response):
        title = response.css('.shici-title::text').extract_first()
        niandai = response.css('.shici-info::text').extract_first().strip()
        shiren = response.css('.shici-info a::text').extract_first().strip()
        content = (response.css('.shici-content::text').getall())
        newContent = ""
        item = TutorialItem()

        for cj in content:
            newContent += cj

        item['_id'] = title + str(random.randint(1, 1000))
        item["title"] = title
        item["niandai"] = niandai
        item["shiren"] = shiren
        item["content"] = newContent
        print(item)
        # yield item